<!DOCTYPE html> <html><!--
 Archive processed by SingleFile 
 url: https://mp.weixin.qq.com/s/ztVsWqAtO1kT9dFZLW3rZg 
 saved date: Thu Nov 21 2019 11:19:09 GMT+0800 (中国标准时间) 
--><meta charset=utf-8>
<meta http-equiv=X-UA-Compatible content="IE=edge">
<meta name=viewport content="width=device-width,initial-scale=1.0,maximum-scale=1.0,user-scalable=0,viewport-fit=cover">
<meta name=apple-mobile-web-app-capable content=yes>
<meta name=apple-mobile-web-app-status-bar-style content=black>
<meta name=format-detection content="telephone=no">
<meta name=description content=公司是做社交相关产品的，社交类产品对搜索功能需求要求就比较高，需要根据用户城市、用户ID昵称等进行搜索。项目>
<meta name=author content>
<meta property=og:title content="ElasticSearch + Canal 开发千万级的实时搜索系统">
<meta property=og:url content="http://mp.weixin.qq.com/s?__biz=MzI3NjU2ODA5Mg==&amp;mid=2247483737&amp;idx=1&amp;sn=7aba98a10984726b9015c34d42379979&amp;chksm=eb72c02adc05493ceeb26ebff5ecddeaed86ad2f93a6e3facf27b91fca1a947c6fb1af5006e4#rd">
<meta property=og:image content="http://mmbiz.qpic.cn/mmbiz_jpg/M7B64fHXISuicW1U0hf4xdVhSPCY5tDdNR6Al0cNuMHFMUwhTOaQEAs5pN5BwAwx7sAoVuiaU3r5XgYQGN6lXPGQ/0?wx_fmt=jpeg">
<meta property=og:description content=公司是做社交相关产品的，社交类产品对搜索功能需求要求就比较高，需要根据用户城市、用户ID昵称等进行搜索。项目>
<meta property=og:site_name content=微信公众平台>
<meta property=og:type content=article>
<meta property=og:article:author content>
<meta property=twitter:card content=summary>
<meta property=twitter:image content="http://mmbiz.qpic.cn/mmbiz_jpg/M7B64fHXISuicW1U0hf4xdVhSPCY5tDdNR6Al0cNuMHFMUwhTOaQEAs5pN5BwAwx7sAoVuiaU3r5XgYQGN6lXPGQ/0?wx_fmt=jpeg">
<meta property=twitter:title content="ElasticSearch + Canal 开发千万级的实时搜索系统">
<meta property=twitter:creator content>
<meta property=twitter:site content=微信公众平台>
<meta property=twitter:description content=公司是做社交相关产品的，社交类产品对搜索功能需求要求就比较高，需要根据用户城市、用户ID昵称等进行搜索。项目>
<title>ElasticSearch + Canal 开发千万级的实时搜索系统</title>
<style media>.rich_media_inner{overflow-wrap:break-word;hyphens:auto}.rich_media_area_primary{padding:calc(20px + env(safe-area-inset-top)) calc(16px + env(safe-area-inset-right)) 12px calc(16px + env(safe-area-inset-left))}.rich_media_area_extra{padding:0 calc(16px + env(safe-area-inset-right)) calc(16px + env(safe-area-inset-bottom)) calc(16px + env(safe-area-inset-left))}html{line-height:1.6}body{color:#333;background-color:#f2f2f2;letter-spacing:.034em}h2{font-weight:400;font-size:16px}*{margin:0;padding:0}a{-webkit-tap-highlight-color:rgba(0,0,0,0)}.rich_media_title{font-size:22px;line-height:1.4;margin-bottom:14px}@supports(-webkit-overflow-scrolling:touch){.rich_media_title{font-weight:700}}.rich_media_meta_list{margin-bottom:22px;line-height:20px;font-size:0;overflow-wrap:break-word;word-break:break-all}.rich_media_meta_list em{font-style:normal}.rich_media_meta{display:inline-block;vertical-align:middle;margin:0 10px 10px 0;font-size:15px;-webkit-tap-highlight-color:rgba(0,0,0,0)}.rich_media_meta_text{color:rgba(0,0,0,0.3)}.rich_media_meta_nickname{position:relative}.rich_media_content{overflow:hidden;color:#333;font-size:17px;overflow-wrap:break-word;hyphens:auto;text-align:justify;z-index:0}.rich_media_content *{max-width:100% !important;box-sizing:border-box !important;overflow-wrap:break-word !important}.rich_media_content p{clear:both;min-height:1em}.rich_media_content .list-paddingleft-2{padding-left:2.2em}@media screen and (min-width:1024px){.rich_media_area_primary_inner,.rich_media_area_extra_inner{max-width:677px;margin-left:auto;margin-right:auto}.rich_media_area_primary{padding-top:32px}}.appmsg_skin_default .rich_media_area_primary{background-color:#fff}.appmsg_style_default .rich_media_tool{padding-top:15px}.read-more__area{margin:30px 0}html,body{height:100%}</style>
<!--[if lt IE 9]>
<link rel="stylesheet" type="text/css" href="//res.wx.qq.com/mmbizwap/zh_CN/htmledition/style/page/appmsg_new/pc492bcc.css">
<![endif]-->
<style id=page/appmsg_new/not_in_mm.css>.weui-flex{display:flex}.weui-flex__item{-webkit-box-flex:1;flex:1 1 0%}html{text-size-adjust:100%}body{line-height:1.6;font-family:-apple-system-font,BlinkMacSystemFont,"Helvetica Neue","PingFang SC","Hiragino Sans GB","Microsoft YaHei UI","Microsoft YaHei",Arial,sans-serif;font-size:16px}body,h2,p,ul{margin:0}a{color:#576b95;text-decoration:none}body,html{-webkit-appearance:none;-webkit-tap-highlight-color:rgba(0,0,0,0)}@media(orientation:portrait){@-webkit-keyframes opr_fade_out{0%{opacity:1}100%{opacity:0}}@-webkit-keyframes opr_fade_in{0%{opacity:0}100%{bottom:0;opacity:1}}}@-webkit-keyframes opr_fade_out{0%{opacity:1}100%{opacity:0}}@-webkit-keyframes opr_fade_in{0%{opacity:0}100%{opacity:1}}@-webkit-keyframes opacity-60-25-0-12{0%{opacity:.25}0.01%{opacity:.25}0.02%{opacity:1}60.01%{opacity:.25}100%{opacity:.25}}@-webkit-keyframes opacity-60-25-1-12{0%{opacity:.25}8.34333%{opacity:.25}8.35333%{opacity:1}68.3433%{opacity:.25}100%{opacity:.25}}@-webkit-keyframes opacity-60-25-2-12{0%{opacity:.25}16.6767%{opacity:.25}16.6867%{opacity:1}76.6767%{opacity:.25}100%{opacity:.25}}@-webkit-keyframes opacity-60-25-3-12{0%{opacity:.25}25.01%{opacity:.25}25.02%{opacity:1}85.01%{opacity:.25}100%{opacity:.25}}@-webkit-keyframes opacity-60-25-4-12{0%{opacity:.25}33.3433%{opacity:.25}33.3533%{opacity:1}93.3433%{opacity:.25}100%{opacity:.25}}@-webkit-keyframes opacity-60-25-5-12{0%{opacity:.270958}41.6767%{opacity:.25}41.6867%{opacity:1}1.67667%{opacity:.25}100%{opacity:.270958}}@-webkit-keyframes opacity-60-25-6-12{0%{opacity:.375125}50.01%{opacity:.25}50.02%{opacity:1}10.01%{opacity:.25}100%{opacity:.375125}}@-webkit-keyframes opacity-60-25-7-12{0%{opacity:.479292}58.3433%{opacity:.25}58.3533%{opacity:1}18.3433%{opacity:.25}100%{opacity:.479292}}@-webkit-keyframes opacity-60-25-8-12{0%{opacity:.583458}66.6767%{opacity:.25}66.6867%{opacity:1}26.6767%{opacity:.25}100%{opacity:.583458}}@-webkit-keyframes opacity-60-25-9-12{0%{opacity:.687625}75.01%{opacity:.25}75.02%{opacity:1}35.01%{opacity:.25}100%{opacity:.687625}}@-webkit-keyframes opacity-60-25-10-12{0%{opacity:.791792}83.3433%{opacity:.25}83.3533%{opacity:1}43.3433%{opacity:.25}100%{opacity:.791792}}@-webkit-keyframes opacity-60-25-11-12{0%{opacity:.895958}91.6767%{opacity:.25}91.6867%{opacity:1}51.6767%{opacity:.25}100%{opacity:.895958}}@-webkit-keyframes loading{0%{transform:rotate3d(0,0,1,0)}100%{transform:rotate3d(0,0,1,360deg)}}@keyframes loading{0%{transform:rotate3d(0,0,1,0)}100%{transform:rotate3d(0,0,1,360deg)}}.article_extend_area{padding:30px 0 0}.article_extend_area:empty{display:none}@supports(-webkit-overflow-scrolling:touch){.reward_button{font-weight:700}}.rich_media_extra{position:relative}.top_banner{background-color:#fff}.ct_mpda_wrp{margin:38px 0 20px}.like_btn{-webkit-appearance:none;-webkit-tap-highlight-color:rgba(0,0,0,0);outline:0;background-color:transparent;border:0;display:inline-block;vertical-align:middle;padding:0;font-size:15px;font-family:inherit;line-height:2.13333;color:#576b95}.like_btn::before{font-size:16px;content:"";display:inline-block;width:1em;height:1.125em;vertical-align:middle;margin-top:-0.25em;margin-right:.05em;background:url("data:image/svg+xml;charset=utf8, %3Csvg width='18' height='20' viewBox='0 0 18 20' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M5.485 3.785l2.506-2.477a1.674 1.674 0 0 1 2.352 0l2.505 2.477 3.423.908a1.653 1.653 0 0 1 1.18 2.026l-.917 3.382.918 3.384a1.652 1.652 0 0 1-1.18 2.024l-3.399.902-2.53 2.493a1.674 1.674 0 0 1-2.352 0l-2.506-2.477-3.423-.908a1.653 1.653 0 0 1-1.18-2.026l.917-3.383-.918-3.392a1.652 1.652 0 0 1 1.18-2.025l3.424-.908zm.836 1.447l.006 2.298c0 .59-.317 1.138-.828 1.438l-2.015 1.143 2.005 1.136c.517.29.838.841.838 1.435l-.006 2.298 2.01-1.156a1.667 1.667 0 0 1 1.675-.003l2.007 1.154-.007-2.302c0-.583.319-1.13.829-1.43l2.014-1.142-2.005-1.136a1.647 1.647 0 0 1-.838-1.435l.007-2.298-2.01 1.156a1.65 1.65 0 0 1-1.67.001L6.321 5.232zm-1.094 2.3L5.22 4.994l-2.878.763a.552.552 0 0 0-.398.674l.77 2.851 2.23-1.264a.573.573 0 0 0 .283-.486zm-.278 4.673L2.714 10.94l-.77 2.84a.553.553 0 0 0 .399.676l2.877.763.007-2.537a.548.548 0 0 0-.278-.476zm3.935 2.57l-2.216 1.274 2.096 2.073c.222.22.583.22.806 0l2.103-2.073-2.214-1.274a.57.57 0 0 0-.575 0zm4.222-2.104l.007 2.538 2.879-.763a.552.552 0 0 0 .398-.674l-.771-2.843-2.23 1.265a.57.57 0 0 0-.283.477zm.279-4.664l2.234 1.266.77-2.84a.553.553 0 0 0-.399-.676l-2.877-.763-.007 2.537c0 .196.107.38.279.476zm-4.501-2.57c.176.104.39.104.566 0l2.215-1.274L9.57 2.09a.574.574 0 0 0-.805 0L6.668 4.163l2.216 1.274z' fill='%23576B95' fill-rule='evenodd'/%3E%3C/svg%3E") 0 0 / 1em no-repeat transparent}.like_num{font-size:15px;margin-left:.2em}.like_comment_wrp{font-size:17px;margin-top:9px;margin-bottom:8px;position:relative;z-index:1}.like_comment_wrp::before,.like_comment_wrp::after{content:"";display:inline-block;width:0;height:0;border-width:0 7px 7px;border-style:dashed dashed solid;border-color:transparent transparent rgba(0,0,0,0.03);position:absolute;top:-7px;right:28px}.like_comment_wrp::after{border-bottom-color:#f7f7f7;top:-6px}.like_comment_inner{background-color:rgba(0,0,0,0.03);border-radius:4px;overflow:hidden;padding:24px 16px;display:flex;-webkit-box-align:center;align-items:center;text-align:center}.like_comment_primary_wrp{font-size:16px;margin-top:9px;margin-bottom:4px;background-color:#fff;z-index:21}.like_comment_primary_wrp::before,.like_comment_primary_wrp::after{content:"";display:inline-block;width:0;height:0;border-width:0 7px 7px;border-style:dashed dashed solid;border-color:transparent transparent #fff;position:absolute;top:-7px;right:28px}.like_comment_primary_wrp::after{border-bottom-color:#fff;top:-6px}.like_comment_primary_wrp.editing{position:fixed;top:10px;bottom:0;left:0;right:0;margin:0}.like_comment_primary_wrp.editing::before,.like_comment_primary_wrp.editing::after{display:none}.like_comment_primary_mask{position:fixed;z-index:20;top:0;left:0;bottom:0;right:0;background-color:rgba(0,0,0,0.2)}@-webkit-keyframes weuiLoading{0%{transform:rotate3d(0,0,1,0)}100%{transform:rotate3d(0,0,1,360deg)}}@keyframes weuiLoading{0%{transform:rotate3d(0,0,1,0)}100%{transform:rotate3d(0,0,1,360deg)}}@-webkit-keyframes slidein{0%{transform:translateX(-50%)}100%{transform:translateX(0)}}@keyframes slidein{0%{transform:translateX(-50%)}100%{transform:translateX(0)}}.mpda_bottom_container{position:relative}.rich_media_tool{overflow:hidden;line-height:32px}.rich_media_tool .meta_primary{float:left}.rich_media_tool .meta_extra{float:right}.rich_media_tool .meta_praise{text-align:right}.media_tool_meta i{vertical-align:0;position:relative;top:1px}.meta_praise{-webkit-tap-highlight-color:rgba(0,0,0,0);outline:0}.meta_praise .praise_num{display:inline-block;vertical-align:top}.meta_praise:hover{cursor:pointer}.icon_praise_gray{background:url("") 0 0 / 100% no-repeat transparent;display:inline-block}.rich_media_tool{font-size:15px}.rich_media_tool .meta_primary{margin-right:16px}.rich_media_tool .meta_extra{margin-left:16px;color:#576b95}.rich_media_tool .meta_praise{min-width:2.5em}.rich_media_tool .meta_praise i{margin-right:5px}.icon_praise_gray{background-image:url("data:image/svg+xml;charset=utf8, %3Csvg width='16' height='16' viewBox='0 0 16 16' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M2.5 6.988h-.003c-.095-.01-.167-.022-.125-.022H1.75c-.343 0-.75.39-.75.7v6.73c0 .31.27.57.611.57H2.5V7.01a.51.51 0 0 1 0-.022zm1 .003a.55.55 0 0 1 0 .02v7.955h7.414c.748 0 1.395-.361 1.773-1.324a37.17 37.17 0 0 0 1.115-2.57c.219-.564.413-1.11.575-1.627.247-.785.413-1.48.484-2.058.073-.595-.565-1.021-1.236-1.021h-4.97l.102-.586.18-1.027.13-.55a35.058 35.058 0 0 0 .245-1.128c.212-1.098-.483-2.019-1.238-2.067-.74-.048-1.1.111-1.104.562-.008 1.276-.45 2.805-1.252 4.129-.357.589-.899.965-1.56 1.16-.217.065-.438.107-.658.132zm6.345-1.625h3.78c1.19 0 2.393.804 2.229 2.143-.08.646-.26 1.397-.523 2.235-.17.54-.37 1.107-.597 1.69a38.158 38.158 0 0 1-1.133 2.61c-.525 1.346-1.557 1.922-2.687 1.922H1.61c-.886 0-1.611-.698-1.611-1.57v-6.73c0-.871.864-1.7 1.75-1.7l.719.009A3.285 3.285 0 0 0 3.876 5.9c.435-.13.769-.361.986-.72.71-1.171 1.102-2.525 1.108-3.618C5.978.338 6.901-.07 8.14.01c1.36.088 2.48 1.57 2.155 3.255a36.012 36.012 0 0 1-.253 1.167l-.124.52-.072.414z' fill='%23576B95' fill-rule='nonzero'/%3E%3C/svg%3E");font-size:16px;width:1em;height:1em;background-size:1em}.praise_num{color:#576b95}a,button{cursor:pointer}.rich_media_extra{overflow:hidden}.rich_media_extra_discuss{padding-top:0}.praise_num:empty{margin-left:-3px}.comment_primary_emotion_panel_wrp{position:absolute;z-index:1;padding-top:8px;padding-bottom:16px}.comment_primary_emotion_panel{background:#fff;box-shadow:rgba(0,0,0,0.16) 0 2px 8px 0;border-radius:4px;width:376px;height:216px;overflow-y:auto}.tips_global_primary{color:rgba(0,0,0,0.3)}.weui-dialog{position:fixed;z-index:5000;top:50%;left:16px;right:16px;transform:translate(0,-50%);background-color:#fff;text-align:center;border-radius:12px;overflow:hidden;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;flex-direction:column;max-height:90%}@media screen and (min-width:352px){.weui-dialog{width:320px;margin:0 auto}}.weui-toast{position:fixed;z-index:5000;width:120px;height:120px;top:40%;left:50%;transform:translate(-50%,-50%);background:rgba(17,17,17,0.7);text-align:center;border-radius:5px;color:#fff;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;flex-direction:column;-webkit-box-align:center;align-items:center;-webkit-box-pack:center;justify-content:center}.weui-mask{position:fixed;z-index:1000;top:0;right:0;left:0;bottom:0;background:rgba(0,0,0,0.6)}.weui-mask_transparent{position:fixed;z-index:1000;top:0;right:0;left:0;bottom:0}@-webkit-keyframes weuiLoading{0%{transform:rotate3d(0,0,1,0)}100%{transform:rotate3d(0,0,1,360deg)}}@keyframes weuiLoading{0%{transform:rotate3d(0,0,1,0)}100%{transform:rotate3d(0,0,1,360deg)}}@media screen and (max-width:1023px){.profile_container{display:none !important}}.weui-desktop-popover{white-space:normal;overflow-wrap:break-word;hyphens:auto;z-index:500;color:#353535;line-height:1.6;background:#fff;border-radius:2px}.weui-desktop-popover::before{content:" ";width:8px;height:8px;background-color:#fff;box-shadow:#d4d4d4 0 2px 10px 0;transform:matrix(0.71,0.71,-0.71,0.71,0,0);position:absolute}.weui-desktop-popover::after{content:" ";background-color:#fff;position:absolute}.weui-desktop-popover_img-text{text-align:center}.weui-desktop-popover_pos-up-center{margin-top:16px}.weui-desktop-popover_pos-up-left::before,.weui-desktop-popover_pos-up-center::before,.weui-desktop-popover_pos-up-right::before{top:-4px}.weui-desktop-popover_pos-up-left::after,.weui-desktop-popover_pos-up-center::after,.weui-desktop-popover_pos-up-right::after{height:10px;top:0;left:0;right:0}.weui-desktop-popover_pos-up-center::before,.weui-desktop-popover_pos-down-center::before{margin-left:-4px}.weui-desktop-popover{position:absolute;padding:14px;box-shadow:none;border:1px solid #d9dadc;width:182px;box-sizing:border-box}.weui-desktop-popover::before{box-shadow:none;border:1px solid #d9dadc}.not_in_mm .rich_media_meta_list{position:relative;z-index:1}.not_in_mm .rich_media_content{position:relative}.not_in_mm .profile_container{width:535px;position:absolute;top:100%;left:0;margin-top:10px;font-size:14px}.not_in_mm .profile_inner{position:relative;padding:30px 22px 36px 144px;background-color:#fff;border:1px solid #d9dadc}.not_in_mm .profile_arrow_wrp{position:absolute;left:22px;top:-8px}.not_in_mm .rich_media_inner{position:relative}.not_in_mm .qr_code_pc_outer{position:fixed;left:0;right:0;top:20px;color:#717375;text-align:center;display:none !important}.not_in_mm .qr_code_pc_inner{position:relative;width:740px;margin-left:auto;margin-right:auto}.not_in_mm .qr_code_pc{position:absolute;right:-140px;top:0;width:140px;padding:16px;border:1px solid #d9dadc;background-color:#fff;overflow-wrap:break-word;word-break:break-all}.not_in_mm .qr_code_pc p{font-size:14px;line-height:20px}.not_in_mm .qr_code_pc_img{width:102px;height:102px}@media screen and (min-width:1024px){.not_in_mm .qr_code_pc_outer{top:32px;display:block !important}}.not_in_mm .qr_code_pc{box-sizing:border-box}</style><link rel="shortcut icon" type=image/x-icon href=""></head>
 <body id=activity-detail class="zh_CN mm_appmsg appmsg_skin_default appmsg_style_default not_in_mm">
 
 
 
<div id=js_article class=rich_media>
 
 <div id=js_top_ad_area class=top_banner></div>
 
 <div class=rich_media_inner>
 
 
 <div id=page-content class=rich_media_area_primary>
 <div class=rich_media_area_primary_inner>
 
 
 
 <div id=img-content>
 
 <h2 class=rich_media_title id=activity-name>
 
 
 
 ElasticSearch + Canal 开发千万级的实时搜索系统
 </h2>
 <div id=meta_content class=rich_media_meta_list>
 
 <span class="rich_media_meta rich_media_meta_nickname" id=profileBt>
 <a href=https://mp.weixin.qq.com/s/ztVsWqAtO1kT9dFZLW3rZg id=js_name>
 Java架构师之路 </a>
 <div id=js_profile_qrcode class=profile_container style=display:none>
 <div class=profile_inner>
 
 
 
 
 
 </div>
 <span class=profile_arrow_wrp id=js_profile_arrow_wrp>
 
 
 </span>
 </div>
 </span>
 <em id=publish_time class="rich_media_meta rich_media_meta_text">2017-04-07</em>
 </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 <div class=rich_media_content id=js_content>
 
 
 
 
 <p><img data-type=gif data-src="http://mmbiz.qpic.cn/mmbiz_gif/M7B64fHXISsYcoUWO1EJTpLKwebCv6fK6SVj0WRJibTN1rQdbpdjPdTsq0oic6RhibicLW7tfr8OibS15bjOvVBYEmg/0?wx_fmt=gif" style="width:100% !important;height:auto !important;visibility:visible !important" data-ratio=0.1 data-w=500 _width=100% class=__bg_gif src="" data-order=0 data-fail=0><p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>公司是做社交相关产品的，社交类产品对搜索功能需求要求就比较高，需要根据用户城市、用户ID昵称等进行搜索。<p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>项目原先的搜索接口采用SQL查询的方式实现，数据库表采用了按城市分表的方式。但随着业务的发展，搜索接口调用频次越来越高，搜索接口压力越来越大，搜索数据库经常崩溃，从而导致搜索功能经常不能使用。<p><img data-ratio=0.6625 data-src="http://mmbiz.qpic.cn/mmbiz_jpg/M7B64fHXISuicW1U0hf4xdVhSPCY5tDdNHN2CmicEe1n3Zoq1efQTiat7RPwaOFZPtNB2AaMTa3aSxFXylMuibUmzg/0?wx_fmt=jpeg" data-type=jpeg data-w=640 height=938 style='border:0;display:block;margin:10px auto;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff;width:100% !important;height:auto !important;visibility:visible !important' width=1417 _width=100% src="" crossorigin=anonymous data-fail=0><p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>从上面的系统架构图可以看出，当用户修改资料时，接口会修改用户库信息，接着触发器会将改变的用户信息写入临时表。定时脚本每隔1分钟扫描一次临时表，将变更的数据写入到搜索库中。当用户再次请求搜索接口时，就可以搜索到最新的数据。<p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>从技术层面分析，原搜索系统的设计有以下缺点：<ul style=list-style-type:square class=list-paddingleft-2><li><p><span style=font-weight:700>搜索信息不实时。</span>当用户修改信息时，需要等待1分钟的时间才能将最新的用户信息同步到搜索数据库中。</p><li><p><span style=font-weight:700>ID、昵称搜索速度慢。</span>按照地区分表的数据库设计是为了减轻数据库压力，保证大部分按照地区搜索的请求能正常响应。但是如果用户按照ID或昵称搜索，那么我们就需要对成千上万个地区表全都搜索一次，这时间复杂度可想而知。很多时候按照昵称和ID搜索速度太慢，需要10多秒才能响应。</p><li><p><span style=font-weight:700>系统稳定性、拓展性以及处理能力差。</span>这可以归结为技术老旧，无法满足业务需求。随着搜索量的提升，对数据库的压力将会越来越大，而MySQL数据库天然不适合用来应对海量的请求。现在已经有更加成熟的ElasticSearch可以用来做搜索方面的业务。</p><li><p><span style=font-weight:700>触发器不便于管理。</span>触发器这种东西不好维护，并且扩展性很差，一旦修改的请求变多，很可能导致整个数据库崩溃（用户库崩溃是很严重的）。</p></ul><p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>我们总结一下新搜索系统需要解决的几个问题：<ul style=list-style-type:square class=list-paddingleft-2><li><p>海量请求。几百万的请求毫无压力，上千万上亿也要可以扛得住。</p><li><p>实时搜索。指的是当一个用户修改了其数据之后，另一个用户能实时地搜索到改用户。</p></ul><p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'><span style=font-weight:700>海量请求。</span>要扛得起海量的搜索请求，可以使用ElasticSearch来实现，它是在Lucene的基础上进行封装的一个开源项目，它将Lucene复杂的原理以及API封装起来，对外提供了一个易用的API接口。ElasticSearch现在已经广泛地被许多公司使用，其中包括：爱奇艺、百姓网、58到家等公司。<p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'><span style=font-weight:700>实时搜索。</span>阿里有一个开源项目Canal，就是用来解决这个问题的，Canal项目利用了MySQL数据库主从同步的原理，将Canal Server模拟成一台需要同步的从库，从而让主库将binlog日志流发送到Canal Server接口。Canal项目对binlog日志的解析进行了封装，我们可以直接得到解析后的数据，而不需要理会binlog的日志格式。而且Canal项目整合了zookeeper，整体实现了高可用，可伸缩性强，是一个不错的解决方案。<p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>经过一段时间的技术预研，我们设计了整个搜索技术架构：<p><img data-ratio=1.5921875 data-src="http://mmbiz.qpic.cn/mmbiz_jpg/M7B64fHXISuicW1U0hf4xdVhSPCY5tDdNEBE2fiaY1LScEBtwWTIOzXrqNggsM20NLoH4jKzxoxYxNNTsIuhlfZw/0?wx_fmt=jpeg" data-type=jpeg data-w=640 height=1978 style='border:0;display:block;margin:10px auto;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff;width:100% !important;height:auto !important;visibility:visible !important' width=1242 _width=100% src= crossorigin=anonymous data-fail=0><p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>从架构图可以看出整个系统分为两大部分：<ul style=list-style-type:square class=list-paddingleft-2><li><p><span style=font-weight:700>Canal数据变更服务平台。</span>这部分负责解析MySQL的binlog日志，并将其解析后的数据封装成特定的对象放到Kafka中。</p><li><p><span style=font-weight:700>Kafka数据消费方。</span>这部分负责消费存放在Kafka中的消息，当消费方拿到具体的用户表变更消息时，将最新的用户信息存放到ES数据仓库中。</p></ul><p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'><br><h2 style='color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;line-height:28px;white-space:normal;background-color:#ffffff'>Canal技术变更基础平台</h2><p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>因为考虑到未来可能有其他项目需要监控数据库某些表的变化，因此我们将Canal获取MySQL数据变更部分做成一个公用的平台。当有其他业务需要增加监控的表时，我们可以直接修改配置文件，重启服务器即可完成添加，极大地提高了开发效率。<p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>在这一部分中，主要分为两大部分：Canal Server 和 Canal Client。<p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'><span style=font-weight:700>Canal Server端。</span>Canal Server伪装成MySQL的一个从库，使主库发送binlog日志给 Canal Server，Canal Server 收到binlog消息之后进行解析，解析完成后将消息直接发送给Canal Client。在Canal Server端可以设置配置文件进行具体scheme（数据库）和table（数据库表）的筛选，从而实现动态地增加对数据库表的监视。<p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'><span style=font-weight:700>Canal Client端。</span>Canal Client端接收到Canal Server的消息后直接将消息存到Kafka指定Partition中，并将最新的binlogid发送给zookeeper集群保存。<p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'><br><h2 style='color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;line-height:28px;white-space:normal;background-color:#ffffff'>Kafka消息消费端</h2><p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>Canal技术变更平台在获取到对应的数据库变更消息后会将其放到指定的Kafka分片里，具体的业务项目需要到指定的Kafka片区里消费对应的数据变更消息，之后根据具体的业务需求进行处理。<p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>因为Canal变化是根据表为最小单位进行地，因此我在实现方面定义了一个以表为处理单位的<code style=font-family:inherit>MsgDealer</code>接口：<pre style=margin-top:0;margin-bottom:0;padding:0;font-family:inherit;white-space:normal;color:#222222;font-size:16px;line-height:28px;background-color:#ffffff><code style=font-family:inherit>public interface MsgDealer { void deal(CanalMsgVo canalMsgVo); }</code></pre><p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>搜索库涉及对5个表的监视，因此我实现了5个对应的处理类：<p><img data-ratio=0.22751677852348992 data-src="http://mmbiz.qpic.cn/mmbiz_jpg/M7B64fHXISuicW1U0hf4xdVhSPCY5tDdNWvcIUKEPI2fYSHOErjmSFIG1ERhYfX8FVPFMX5dEdD6kakDCgT2l2w/0?wx_fmt=jpeg" data-type=jpeg data-w=1490 height=339 style='border:0;display:block;margin:10px auto;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff;width:100% !important;height:auto !important;visibility:visible !important' width=1490 _width=100% src="" crossorigin=anonymous data-fail=0><p style='margin-top:18px;color:#222222;font-family:"PingFang SC","Hiragino Sans GB","Microsoft YaHei","WenQuanYi Micro Hei","Helvetica Neue",Arial,sans-serif;font-size:16px;line-height:28px;white-space:normal;background-color:#ffffff'>针对不同表的数据变化，自动调用不同的实现类进行处理。<p><img data-s=300,640 data-type=jpeg data-src="http://mmbiz.qpic.cn/mmbiz_jpg/M7B64fHXISsYcoUWO1EJTpLKwebCv6fKoGT2uBQZj53ibwNuPfEyStQ50UVW13A7V8CIBMDyM62dRic52wTU5HGw/0?wx_fmt=jpeg" style="width:100% !important;height:auto !important;visibility:visible !important" data-ratio=0.3333333333333333 data-w=600 _width=100% src="" crossorigin=anonymous data-fail=0><p><br></p>
 </div>
 
 
 <div class=ct_mpda_wrp id=js_sponsor_ad_area style=display:none></div>
 
 <div class=read-more__area id=js_more_read_area style=display:none>
 
 </div>
 </div>
 
 
 
 <ul id=js_hotspot_area class=article_extend_area></ul>
 
 
<div class=rich_media_tool id=js_toobar3>
 <div class=weui-flex>
 <div class=weui-flex__item>
 
 <div id=js_read_area3 class="media_tool_meta tips_global_primary meta_primary" style=display:none>
 <span id=readTxt>阅读</span>
 <span id=readNum3></span>
 </div>
 </div>
 <span style=display:none class="media_tool_meta meta_extra meta_praise" id=like_old>
 <i class=icon_praise_gray></i><span class=praise_num id=likeNum_old></span>
 </span>
 
 <span style=visibility:hidden class="media_tool_meta meta_extra meta_like" id=like3>
 <button class=like_btn id=js_like_btn> 
 <span id=js_like_wording> 在看</span><span class=like_num id=likeNum3></span>
 </button>
 </span>
 
 </div>
</div>
 
 <div class=like_comment_wrp id=js_like_comment style=display:none>
 <div class=like_comment_inner>
 
 
 </div>
 </div> 
 <div style=display:none id=wow_close_inform>
 <div class=weui-mask></div>
 <div class=weui-dialog>
 
 
 
 </div>
 </div>
<div id=js_like_toast style=display:none>
 <div class=weui-mask_transparent></div>
 <div class=weui-toast>
 
 
 </div>
</div>
<div style=display:none id=js_comment_panel>
 <div class="like_comment_primary_wrp editing" id=js_comment_wrp>
 
 </div> 
 <div class=like_comment_primary_mask id=js_mask_2></div>
</div>
<div id=js_loading style=display:none>
 <div class=weui-mask_transparent></div>
 <div class=weui-toast>
 
 
 </div>
</div>
 </div>
 </div>
 <div class="rich_media_area_primary sougou" id=sg_tj style=display:none></div>
 
 <div class=rich_media_area_extra>
 <div class=rich_media_area_extra_inner>
 
 <div id=js_share_appmsg>
 </div>
 
 
 <div class=mpda_bottom_container id=js_bottom_ad_area style=display:none></div>
 
 <div id=js_iframetest style=display:none></div>
 
 <div class="rich_media_extra rich_media_extra_discuss" id=js_cmt_container style=display:none>
 
 
 <div class=discuss_mod id=js_friend_cmt_area style=display:none>
 
 
 
 </div>
 <div class=discuss_mod id=js_cmt_area style=display:none>
 </div>
 </div>
 </div>
 </div>
 
 <div id=js_pc_qr_code class=qr_code_pc_outer style=display:block;>
 <div class=qr_code_pc_inner>
 <div class=qr_code_pc>
 <img id=js_pc_qr_code_img class=qr_code_pc_img src="">
 <p>微信扫一扫<br>关注该公众号</p>
 </div>
 </div>
 </div>
 </div>
</div>
<div id=js_pc_weapp_code class="weui-desktop-popover weui-desktop-popover_pos-up-center weui-desktop-popover_img-text" style=display:none>
 <div class=weui-desktop-popover__content>
 
 </div>
</div>
<div id=js_minipro_dialog style=display:none>
 <div class=weui-mask></div>
 <div class="weui-dialog weui-dialog_link">
 
 
 
 
 </div>
</div>
<div id=js_link_dialog style=display:none>
 <div class=weui-mask></div>
 <div class="weui-dialog weui-dialog_link">
 
 
 
 
 </div>
</div>
<div class=comment_primary_emotion_panel_wrp id=js_emotion_panel_pc style=display:none>
 <div class=comment_primary_emotion_panel>
 
 </div>
</div>
<div class=weui-dialog__wrp id=js_alert_panel style=display:none>
 <div class=weui-mask></div>
 <div class=weui-dialog>
 
 
 </div>
</div>
<div id=js_weapp_without_auth_dialog style=display:none>
 <div class=weui-mask></div>
 <div class="weui-dialog weui-dialog_link">
 
 
 </div>
</div>
 
 
 
 
 
 
